from pythonProject.spider.data_storage import DataStorage
from pythonProject.spider.html_downloader import HtmlDownloader
from pythonProject.spider.html_parser import HtmlParser
from pythonProject.spider.url_manager import UrlManager


class SpiderMain:

    def __init__(self):
        """
        初始化方法，主要是将其他组件实例化
        """
        self.url_manager = UrlManager()
        self.html_downloader = HtmlDownloader()
        self.html_parser = HtmlParser()
        self.data_storage = DataStorage()

    def start(self):
        """
        爬虫的主启动方法
        :return:
        """
        self.url_manager.add_new_url("http://127.0.0.1:8848/xiaomi-master/index.html")
        # 从url管理器里面获取url
        url = self.url_manager.get_new_url()
        # 将获取到的url使用下载器进行下载
        html = self.html_downloader.download(url)
        # 将html进行解析
        result = self.html_parser.parser(html)
        # 数据存储
        self.data_storage.storage(result)


if __name__ == "__main__":
    main = SpiderMain()
    main.start()
